{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Copyright 2015 Google Inc. All Rights Reserved.\n",
    "#\n",
    "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
    "# you may not use this file except in compliance with the License.\n",
    "# You may obtain a copy of the License at\n",
    "#\n",
    "#     http://www.apache.org/licenses/LICENSE-2.0\n",
    "#\n",
    "# Unless required by applicable law or agreed to in writing, software\n",
    "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
    "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
    "# See the License for the specific language governing permissions and\n",
    "# limitations under the License.\n",
    "# ==============================================================================\n",
    "\n",
    "\"\"\"Trains and Evaluates the MNIST network using a feed dictionary.\"\"\"\n",
    "# pylint: disable=missing-docstring\n",
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "import os.path\n",
    "import time\n",
    "\n",
    "import numpy\n",
    "from six.moves import xrange  # pylint: disable=redefined-builtin\n",
    "import tensorflow as tf\n",
    "\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "from tensorflow.examples.tutorials.mnist import mnist\n",
    "\n",
    "\n",
    "# Basic model parameters as external flags.\n",
    "flags = tf.app.flags\n",
    "FLAGS = flags.FLAGS\n",
    "flags.DEFINE_float('learning_rate', 0.01, 'Initial learning rate.')\n",
    "flags.DEFINE_integer('max_steps', 2000, 'Number of steps to run trainer.')\n",
    "flags.DEFINE_integer('hidden1', 128, 'Number of units in hidden layer 1.')\n",
    "flags.DEFINE_integer('hidden2', 32, 'Number of units in hidden layer 2.')\n",
    "flags.DEFINE_integer('batch_size', 100, 'Batch size.  '\n",
    "                     'Must divide evenly into the dataset sizes.')\n",
    "flags.DEFINE_string('train_dir', 'data', 'Directory to put the training data.')\n",
    "flags.DEFINE_boolean('fake_data', False, 'If true, uses fake data '\n",
    "                     'for unit testing.')\n",
    "\n",
    "\n",
    "def placeholder_inputs(batch_size):\n",
    "  \"\"\"Generate placeholder variables to represent the input tensors.\n",
    "  These placeholders are used as inputs by the rest of the model building\n",
    "  code and will be fed from the downloaded data in the .run() loop, below.\n",
    "  Args:\n",
    "    batch_size: The batch size will be baked into both placeholders.\n",
    "  Returns:\n",
    "    images_placeholder: Images placeholder.\n",
    "    labels_placeholder: Labels placeholder.\n",
    "  \"\"\"\n",
    "  # Note that the shapes of the placeholders match the shapes of the full\n",
    "  # image and label tensors, except the first dimension is now batch_size\n",
    "  # rather than the full size of the train or test data sets.\n",
    "  images_placeholder = tf.placeholder(tf.float32, shape=(batch_size,\n",
    "                                                         mnist.IMAGE_PIXELS))\n",
    "  labels_placeholder = tf.placeholder(tf.int32, shape=(batch_size))\n",
    "  return images_placeholder, labels_placeholder\n",
    "\n",
    "\n",
    "def fill_feed_dict(data_set, images_pl, labels_pl):\n",
    "  \"\"\"Fills the feed_dict for training the given step.\n",
    "  A feed_dict takes the form of:\n",
    "  feed_dict = {\n",
    "      <placeholder>: <tensor of values to be passed for placeholder>,\n",
    "      ....\n",
    "  }\n",
    "  Args:\n",
    "    data_set: The set of images and labels, from input_data.read_data_sets()\n",
    "    images_pl: The images placeholder, from placeholder_inputs().\n",
    "    labels_pl: The labels placeholder, from placeholder_inputs().\n",
    "  Returns:\n",
    "    feed_dict: The feed dictionary mapping from placeholders to values.\n",
    "  \"\"\"\n",
    "  # Create the feed_dict for the placeholders filled with the next\n",
    "  # `batch size ` examples.\n",
    "  images_feed, labels_feed = data_set.next_batch(FLAGS.batch_size,\n",
    "                                                 FLAGS.fake_data)\n",
    "  feed_dict = {\n",
    "      images_pl: images_feed,\n",
    "      labels_pl: labels_feed,\n",
    "  }\n",
    "  return feed_dict\n",
    "\n",
    "\n",
    "def do_eval(sess,\n",
    "            eval_correct,\n",
    "            images_placeholder,\n",
    "            labels_placeholder,\n",
    "            data_set):\n",
    "  \"\"\"Runs one evaluation against the full epoch of data.\n",
    "  Args:\n",
    "    sess: The session in which the model has been trained.\n",
    "    eval_correct: The Tensor that returns the number of correct predictions.\n",
    "    images_placeholder: The images placeholder.\n",
    "    labels_placeholder: The labels placeholder.\n",
    "    data_set: The set of images and labels to evaluate, from\n",
    "      input_data.read_data_sets().\n",
    "  \"\"\"\n",
    "  # And run one epoch of eval.\n",
    "  true_count = 0  # Counts the number of correct predictions.\n",
    "  steps_per_epoch = data_set.num_examples // FLAGS.batch_size\n",
    "  num_examples = steps_per_epoch * FLAGS.batch_size\n",
    "  for step in xrange(steps_per_epoch):\n",
    "    feed_dict = fill_feed_dict(data_set,\n",
    "                               images_placeholder,\n",
    "                               labels_placeholder)\n",
    "    true_count += sess.run(eval_correct, feed_dict=feed_dict)\n",
    "  precision = true_count / num_examples\n",
    "  print('  Num examples: %d  Num correct: %d  Precision @ 1: %0.04f' %\n",
    "        (num_examples, true_count, precision))\n",
    "\n",
    "\n",
    "def run_training():\n",
    "  \"\"\"Train MNIST for a number of steps.\"\"\"\n",
    "  # Get the sets of images and labels for training, validation, and\n",
    "  # test on MNIST.\n",
    "  data_sets = input_data.read_data_sets(FLAGS.train_dir, FLAGS.fake_data)\n",
    "\n",
    "  # Tell TensorFlow that the model will be built into the default Graph.\n",
    "  with tf.Graph().as_default():\n",
    "    # Generate placeholders for the images and labels.\n",
    "    images_placeholder, labels_placeholder = placeholder_inputs(\n",
    "        FLAGS.batch_size)\n",
    "\n",
    "    # Build a Graph that computes predictions from the inference model.\n",
    "    logits = mnist.inference(images_placeholder,\n",
    "                             FLAGS.hidden1,\n",
    "                             FLAGS.hidden2)\n",
    "\n",
    "    # Add to the Graph the Ops for loss calculation.\n",
    "    loss = mnist.loss(logits, labels_placeholder)\n",
    "\n",
    "    # Add to the Graph the Ops that calculate and apply gradients.\n",
    "    train_op = mnist.training(loss, FLAGS.learning_rate)\n",
    "\n",
    "    # Add the Op to compare the logits to the labels during evaluation.\n",
    "    eval_correct = mnist.evaluation(logits, labels_placeholder)\n",
    "\n",
    "    # Build the summary operation based on the TF collection of Summaries.\n",
    "    summary_op = tf.merge_all_summaries()\n",
    "\n",
    "    # Create a saver for writing training checkpoints.\n",
    "    saver = tf.train.Saver()\n",
    "\n",
    "    # Create a session for running Ops on the Graph.\n",
    "    sess = tf.Session()\n",
    "\n",
    "    # Run the Op to initialize the variables.\n",
    "    init = tf.initialize_all_variables()\n",
    "    sess.run(init)\n",
    "\n",
    "    # Instantiate a SummaryWriter to output summaries and the Graph.\n",
    "    summary_writer = tf.train.SummaryWriter(FLAGS.train_dir,\n",
    "                                            graph_def=sess.graph_def)\n",
    "\n",
    "    # And then after everything is built, start the training loop.\n",
    "    for step in xrange(FLAGS.max_steps):\n",
    "      start_time = time.time()\n",
    "\n",
    "      # Fill a feed dictionary with the actual set of images and labels\n",
    "      # for this particular training step.\n",
    "      feed_dict = fill_feed_dict(data_sets.train,\n",
    "                                 images_placeholder,\n",
    "                                 labels_placeholder)\n",
    "\n",
    "      # Run one step of the model.  The return values are the activations\n",
    "      # from the `train_op` (which is discarded) and the `loss` Op.  To\n",
    "      # inspect the values of your Ops or variables, you may include them\n",
    "      # in the list passed to sess.run() and the value tensors will be\n",
    "      # returned in the tuple from the call.\n",
    "      _, loss_value = sess.run([train_op, loss],\n",
    "                               feed_dict=feed_dict)\n",
    "\n",
    "      duration = time.time() - start_time\n",
    "\n",
    "      # Write the summaries and print an overview fairly often.\n",
    "      if step % 100 == 0:\n",
    "        # Print status to stdout.\n",
    "        print('Step %d: loss = %.2f (%.3f sec)' % (step, loss_value, duration))\n",
    "        # Update the events file.\n",
    "        summary_str = sess.run(summary_op, feed_dict=feed_dict)\n",
    "        summary_writer.add_summary(summary_str, step)\n",
    "\n",
    "      # Save a checkpoint and evaluate the model periodically.\n",
    "      if (step + 1) % 1000 == 0 or (step + 1) == FLAGS.max_steps:\n",
    "        saver.save(sess, FLAGS.train_dir, global_step=step)\n",
    "        # Evaluate against the training set.\n",
    "        print('Training Data Eval:')\n",
    "        do_eval(sess,\n",
    "                eval_correct,\n",
    "                images_placeholder,\n",
    "                labels_placeholder,\n",
    "                data_sets.train)\n",
    "        # Evaluate against the validation set.\n",
    "        print('Validation Data Eval:')\n",
    "        do_eval(sess,\n",
    "                eval_correct,\n",
    "                images_placeholder,\n",
    "                labels_placeholder,\n",
    "                data_sets.validation)\n",
    "        # Evaluate against the test set.\n",
    "        print('Test Data Eval:')\n",
    "        do_eval(sess,\n",
    "                eval_correct,\n",
    "                images_placeholder,\n",
    "                labels_placeholder,\n",
    "                data_sets.test)\n",
    "\n",
    "\n",
    "def main(_):\n",
    "  run_training()\n",
    "\n",
    "\n",
    "if __name__ == '__main__':\n",
    "  tf.app.run()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
